import requests
import json
import re
import os
import sys
from lxml import etree

cwd = os.getcwd()
sys.path.append(cwd)
print(cwd)

class ImageToutiao(object):
    url = ' https://m.toutiao.com/search?keyword=%E7%BB%93%E8%B4%A6%E5%8D%95&pd=atlas&source=input&traffic_source=&original_source=&in_tfs=&in_ogs='
    # 匹配图片url的xpath，如：https://tse1-mm.cn.bing.net/th/id/OIP.73Ob4lcxFaLaxTKil_dnJQAAAA?w=190&h=143&c=7&o=5&pid=1.7
    img_url_pattern = 'background-image: url(".*?");'
    save_dir = './toutiao' # 文件保存的路径
    count = 0

    # 初始化
    def __init__(self, word):
        self.word = word
        self.dir_path = os.path.join(self.save_dir, word)
        self.params = {
            'keyword': word,
            'pd': 'atlas',
            'source': 'input',
            'count': '20',
            'offset': '0',
            'format': 'raw_json',
            'search_id': '202004241724240101940990320900BF9C',
        }
        self.headers = {
            'Accept': 'application/json',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'Connection': 'keep-alive',
            'Cookie': 'tt_webid=6818854656232474120; SLARDAR_WEB_ID=6818854656232474120; csrftoken=8d0ab5b67409cd1062678f414f5e5cd7; W2atIF=1; _ga=GA1.2.967247976.1587718170; _gid=GA1.2.1398972959.1587718170; __tasessionId=e7qt9tlvv1587718178496; WIN_WH=375_812',
            'Host': 'm.toutiao.com',
            'Sec-Fetch-Dest': 'document',
            'Sec-Fetch-Mode': 'navigate',
            'Sec-Fetch-Site': 'none',
            'Sec-Fetch-User': '?1',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1',
        }
        # 创建保存图片的文件夹
        self.folder_exist(self.dir_path)
        self.last_img_url = ''
        self.current_page = 0
    
    # 请求包含图片url的网页
    def parse(self):
        while True:
            self.params['offset'] = str(self.current_page*20)
            response = requests.get(url=self.url, headers=self.headers, params=self.params)
            data = response.json()['data']
            for img_info in data:
                img_url = img_info['img_url']
                self.download(img_url)
            self.current_page += 1

    # 下载一张图片
    def download(self, img_url, img_type='jpg'):
        self.count += 1
        print('正在下载第%d张图片...'%self.count, img_url)
        response = requests.get(img_url)
        img_name = img_url.split('/')[-1]
        img_path = os.path.join(self.dir_path, img_name)
        with open(img_path, 'wb') as f:
            f.write(response.content)

    def folder_exist(self, dir_path):
        '''
        1. 作用:判断文件夹路径是否存在,不存在则创建
        2. 参数:dir_path:文件夹路径
        3. 返回值:None
        '''
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)

if __name__ == '__main__':
    image = ImageToutiao('黑色牌照')
    image.parse()